package bundle2name

import us.codecraft.webmagic.Page
import us.codecraft.webmagic.Spider
import us.codecraft.webmagic.processor.PageProcessor

val anzhiUrls = arrayOf("http://www.anzhi.com/sort_49_1_hot.html","http://www.anzhi.com/sort_39_1_hot.html",
        "http://www.anzhi.com/sort_40_1_hot.html","http://www.anzhi.com/sort_41_1_hot.html",
        "http://www.anzhi.com/sort_42_1_hot.html","http://www.anzhi.com/sort_43_1_hot.html",
        "http://www.anzhi.com/sort_44_1_hot.html","http://www.anzhi.com/sort_45_1_hot.html",
        "http://www.anzhi.com/sort_46_1_hot.html","http://www.anzhi.com/sort_47_1_hot.html",
        "http://www.anzhi.com/sort_48_1_hot.html","http://www.anzhi.com/sort_50_1_hot.html",
        "http://www.anzhi.com/sort_51_1_hot.html","http://www.anzhi.com/sort_52_1_hot.html",
        "http://www.anzhi.com/sort_53_1_hot.html","http://www.anzhi.com/sort_54_1_hot.html",
        "http://www.anzhi.com/sort_55_1_hot.html","http://www.anzhi.com/sort_21_1_hot.html",
        "http://www.anzhi.com/sort_14_1_hot.html","http://www.anzhi.com/sort_15_1_hot.html",
        "http://www.anzhi.com/sort_16_1_hot.html","http://www.anzhi.com/sort_19_1_hot.html",
        "http://www.anzhi.com/sort_20_1_hot.html","http://www.anzhi.com/sort_24_1_hot.html",
        "http://www.anzhi.com/sort_56_1_hot.html","http://www.anzhi.com/sort_57_1_hot.html")
fun main(args: Array<String>) {
    val sp = Spider.create(AnzhiPP())
            .addUrl(*anzhiUrls)
            .addPipeline(FilePipeline())
            .thread(16)
    sp.run()
}

class AnzhiPP : PP() {
    val P = Regex(".+?_(.+?)\\.html")
    override fun process(page: Page) {
        val apps = page.html.xpath("//div[@class='app_list border_three']/ul/li")
        apps?.let {
            val ps = mutableListOf<BundleName>()
            it.nodes().forEach {
                val name = it.xpath("//span[@class='app_name']/a/text()").get()
                val href = it.xpath("//span[@class='app_name']/a/@href").get()
                val mr = P.find(href)
                mr?.let {
                    it.groupValues.getOrNull(1)?.let {
                        ps.add(BundleName(it,name))
                    }
                }
            }
            page.putField("ps",ps)
            val pager = page.html.xpath("//div[@class='app_list border_three']" +
                    "/div[@class='pagebars']/a[@class='next']/@href").get()
            page.addTargetRequest(pager)
        }
    }

}